





<!DOCTYPE html>
<html class="writer-html5" lang="zh-CN" >
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Simple Matrix Multiply &mdash; tvm 0.8.dev1982 文档</title>
  

  
  <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
  <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../../../_static/gallery.css" type="text/css" />
  <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../../../_static/css/tlcpack_theme.css" type="text/css" />

  
  
    <link rel="shortcut icon" href="../../../_static/tvm-logo-square.png"/>
  

  
  
  
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../../../" src="../../../_static/documentation_options.js"></script>
        <script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script>
        <script src="../../../_static/jquery.js"></script>
        <script src="../../../_static/underscore.js"></script>
        <script src="../../../_static/doctools.js"></script>
        <script src="../../../_static/translations.js"></script>
    
    <script type="text/javascript" src="../../../_static/js/theme.js"></script>

    
    <script type="text/javascript" src="../../../_static/js/tlcpack_theme.js"></script>
    <link rel="index" title="索引" href="../../../genindex.html" />
    <link rel="search" title="搜索" href="../../../search.html" />
    <link rel="next" title="从 VTA 开始" href="vta_get_started.html" />
    <link rel="prev" title="VTA教程" href="index.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    
<header class="header">
    <div class="innercontainer">
      <div class="headerInner d-flex justify-content-between align-items-center">
          <div class="headerLogo">
               <a href="https://tvm.apache.org/"><img src=https://tvm.apache.org/assets/images/logo.svg alt="logo"></a>
          </div>

          <div id="headMenu" class="headerNav">
            <button type="button" id="closeHeadMenu" class="navCloseBtn"><img src="../../../_static/img/close-icon.svg" alt="Close"></button>
             <ul class="nav">
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/community>Community</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/download>Download</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/vta>VTA</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/blog>Blog</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/docs>Docs</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvmconf.org>Conference</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://github.com/apache/tvm/>Github</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvmchinese.github.io/declaration_zh_CN.html>About-Translators</a>
                </li>
             </ul>
               <div class="responsivetlcdropdown">
                 <button type="button" class="btn-link">
                   ASF
                 </button>
                 <ul>
                     <li>
                       <a href=https://apache.org/>Apache Homepage</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/licenses/>License</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/sponsorship.html>Sponsorship</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/security/>Security</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/thanks.html>Thanks</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/events/current-event>Events</a>
                     </li>
                     <li>
                       <a href=https://www.zhihu.com/column/c_1429578595417563136>Zhihu</a>
                     </li>
                 </ul>
               </div>
          </div>
            <div class="responsiveMenuIcon">
              <button type="button" id="menuBtn" class="btn-menu"><img src="../../../_static/img/menu-icon.svg" alt="Menu Icon"></button>
            </div>

            <div class="tlcDropdown">
              <div class="dropdown">
                <button type="button" class="btn-link dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
                  ASF
                </button>
                <div class="dropdown-menu dropdown-menu-right">
                  <ul>
                     <li>
                       <a href=https://apache.org/>Apache Homepage</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/licenses/>License</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/sponsorship.html>Sponsorship</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/security/>Security</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/thanks.html>Thanks</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/events/current-event>Events</a>
                     </li>
                     <li>
                       <a href=https://www.zhihu.com/column/c_1429578595417563136>Zhihu</a>
                     </li>
                  </ul>
                </div>
              </div>
          </div>
       </div>
    </div>
 </header>
 
    <nav data-toggle="wy-nav-shift" class="wy-nav-side fixed">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../../../index.html">
          

          
            
            <img src="../../../_static/tvm-logo-small.png" class="logo" alt="Logo"/>
          
          </a>

          
            
            
                <div class="version">
                  0.8.dev1982
                </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption" role="heading"><span class="caption-text">如何开始</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../install/index.html">安装 TVM</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../contribute/index.html">贡献者指南</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">用户引导</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../tutorial/index.html">User Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../how_to/index.html">How To Guides</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">开发者引导</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/tutorial/index.html">Developer Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/how_to/how_to.html">开发者指南</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">架构指南</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../arch/index.html">Design and Architecture</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">主题引导</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../microtvm/index.html">microTVM：裸机使用TVM</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">VTA: Versatile Tensor Accelerator</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="../install.html">VTA安装指南</a></li>
<li class="toctree-l2"><a class="reference internal" href="../dev/index.html">VTA设计与开发指南</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">VTA教程</a><ul class="current">
<li class="toctree-l3 current"><a class="current reference internal" href="#">Simple Matrix Multiply</a><ul>
<li class="toctree-l4"><a class="reference internal" href="#rpc-setup">RPC设置</a></li>
<li class="toctree-l4"><a class="reference internal" href="#computation-declaration">Computation Declaration</a></li>
<li class="toctree-l4"><a class="reference internal" href="#scheduling-the-computation">Scheduling the Computation</a></li>
<li class="toctree-l4"><a class="reference internal" href="#tvm-compilation">TVM Compilation</a></li>
<li class="toctree-l4"><a class="reference internal" href="#running-the-function">Running the Function</a></li>
<li class="toctree-l4"><a class="reference internal" href="#verifying-correctness">Verifying Correctness</a></li>
<li class="toctree-l4"><a class="reference internal" href="#summary">总结</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="vta_get_started.html">从 VTA 开始</a></li>
<li class="toctree-l3"><a class="reference internal" href="index.html#compile-deep-learning-models">编译深度学习模型</a></li>
<li class="toctree-l3"><a class="reference internal" href="index.html#optimize-tensor-operators">优化张量算子</a></li>
<li class="toctree-l3"><a class="reference internal" href="index.html#auto-tuning">自动调整</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../index.html#literature">Literature</a></li>
</ul>
</li>
</ul>
<p class="caption" role="heading"><span class="caption-text">参考指南</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../reference/langref/index.html">语言参考</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../reference/api/python/index.html">Python API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../reference/api/links.html">Other APIs</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../reference/publications.html">Publications</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../genindex.html">索引</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
      
      <nav class="wy-nav-top" aria-label="top navigation" data-toggle="wy-nav-top">
        
            <div class="togglemenu">

            </div>
            <div class="nav-content">
              <!-- tvm -->
              Table of content
            </div>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        

          




















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../../../index.html">Docs</a> <span class="br-arrow">></span></li>
        
          <li><a href="../index.html">VTA: Versatile Tensor Accelerator</a> <span class="br-arrow">></span></li>
        
          <li><a href="index.html">VTA教程</a> <span class="br-arrow">></span></li>
        
      <li>Simple Matrix Multiply</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
            <a href="../../../_sources/topic/vta/tutorials/matrix_multiply.rst.txt" rel="nofollow"> <img src="../../../_static//img/source.svg" alt="viewsource"/></a>
          
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="sphx-glr-download-link-note admonition note">
<p class="admonition-title">注解</p>
<p>Click <a class="reference internal" href="#sphx-glr-download-topic-vta-tutorials-matrix-multiply-py"><span class="std std-ref">here</span></a> to download the full example code</p>
</div>
<div class="sphx-glr-example-title section" id="simple-matrix-multiply">
<span id="basic-mat-mult"></span><span id="sphx-glr-topic-vta-tutorials-matrix-multiply-py"></span><h1>Simple Matrix Multiply<a class="headerlink" href="#simple-matrix-multiply" title="永久链接至标题">¶</a></h1>
<p><strong>作者</strong>: <a class="reference external" href="https://homes.cs.washington.edu/~moreau/">Thierry Moreau</a></p>
<p>In this tutorial, we will build on top of the <a class="reference internal" href="vta_get_started.html#vta-get-started"><span class="std std-ref">从 VTA 开始</span></a> tutorial
and introduce additional concepts required to implement matrix multiplication
on VTA with the TVM workflow.</p>
<div class="section" id="rpc-setup">
<h2>RPC设置<a class="headerlink" href="#rpc-setup" title="永久链接至标题">¶</a></h2>
<p>We start by programming the Pynq’s FPGA and building its RPC runtime
as we did in the VTA introductory tutorial.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">absolute_import</span><span class="p">,</span> <span class="n">print_function</span>

<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">tvm</span>
<span class="kn">from</span> <span class="nn">tvm</span> <span class="k">import</span> <span class="n">te</span>
<span class="kn">import</span> <span class="nn">vta</span>
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="kn">from</span> <span class="nn">tvm</span> <span class="k">import</span> <span class="n">rpc</span>
<span class="kn">from</span> <span class="nn">tvm.contrib</span> <span class="k">import</span> <span class="n">utils</span>
<span class="kn">from</span> <span class="nn">vta.testing</span> <span class="k">import</span> <span class="n">simulator</span>

<span class="c1"># Load VTA parameters from the 3rdparty/vta-hw/config/vta_config.json file</span>
<span class="n">env</span> <span class="o">=</span> <span class="n">vta</span><span class="o">.</span><span class="n">get_env</span><span class="p">()</span>

<span class="c1"># We read the Pynq RPC host IP address and port number from the OS environment</span>
<span class="n">host</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;VTA_RPC_HOST&quot;</span><span class="p">,</span> <span class="s2">&quot;192.168.2.99&quot;</span><span class="p">)</span>
<span class="n">port</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;VTA_RPC_PORT&quot;</span><span class="p">,</span> <span class="s2">&quot;9091&quot;</span><span class="p">))</span>

<span class="c1"># We configure both the bitstream and the runtime system on the Pynq</span>
<span class="c1"># to match the VTA configuration specified by the vta_config.json file.</span>
<span class="k">if</span> <span class="n">env</span><span class="o">.</span><span class="n">TARGET</span> <span class="o">==</span> <span class="s2">&quot;pynq&quot;</span> <span class="ow">or</span> <span class="n">env</span><span class="o">.</span><span class="n">TARGET</span> <span class="o">==</span> <span class="s2">&quot;de10nano&quot;</span><span class="p">:</span>

    <span class="c1"># Make sure that TVM was compiled with RPC=1</span>
    <span class="k">assert</span> <span class="n">tvm</span><span class="o">.</span><span class="n">runtime</span><span class="o">.</span><span class="n">enabled</span><span class="p">(</span><span class="s2">&quot;rpc&quot;</span><span class="p">)</span>
    <span class="n">remote</span> <span class="o">=</span> <span class="n">rpc</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">port</span><span class="p">)</span>

    <span class="c1"># Reconfigure the JIT runtime</span>
    <span class="n">vta</span><span class="o">.</span><span class="n">reconfig_runtime</span><span class="p">(</span><span class="n">remote</span><span class="p">)</span>

    <span class="c1"># Program the FPGA with a pre-compiled VTA bitstream.</span>
    <span class="c1"># You can program the FPGA with your own custom bitstream</span>
    <span class="c1"># by passing the path to the bitstream file instead of None.</span>
    <span class="n">vta</span><span class="o">.</span><span class="n">program_fpga</span><span class="p">(</span><span class="n">remote</span><span class="p">,</span> <span class="n">bitstream</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>

<span class="c1"># In simulation mode, host the RPC server locally.</span>
<span class="k">elif</span> <span class="n">env</span><span class="o">.</span><span class="n">TARGET</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;sim&quot;</span><span class="p">,</span> <span class="s2">&quot;tsim&quot;</span><span class="p">]:</span>
    <span class="n">remote</span> <span class="o">=</span> <span class="n">rpc</span><span class="o">.</span><span class="n">LocalSession</span><span class="p">()</span>
</pre></div>
</div>
</div>
<div class="section" id="computation-declaration">
<h2>Computation Declaration<a class="headerlink" href="#computation-declaration" title="永久链接至标题">¶</a></h2>
<p>In this example we describe a simple matrix multiplication addition, which
requires multiple computation stages, as shown in the dataflow diagram below.
First we describe the input tensors <code class="code docutils literal notranslate"><span class="pre">A</span></code> and <code class="code docutils literal notranslate"><span class="pre">B</span></code> that are living
in main memory.
Second, we need to declare intermediate tensors <code class="code docutils literal notranslate"><span class="pre">A_buf</span></code> and
<code class="code docutils literal notranslate"><span class="pre">B_buf</span></code>, which will live in VTA’s on-chip buffers.
Having this extra computational stage allows us to explicitly
stage cached reads and writes.
Third, we describe the matrix multiplication computation over
<code class="code docutils literal notranslate"><span class="pre">A_buf</span></code> and <code class="code docutils literal notranslate"><span class="pre">B_buf</span></code> to produce the product matrix <code class="code docutils literal notranslate"><span class="pre">C_buf</span></code>.
The last operation is a cast and copy back to DRAM, into results tensor
<code class="code docutils literal notranslate"><span class="pre">C</span></code>.</p>
<img alt="https://raw.githubusercontent.com/uwsampl/web-data/main/vta/tutorial/gemm_dataflow.png" class="align-center" src="https://raw.githubusercontent.com/uwsampl/web-data/main/vta/tutorial/gemm_dataflow.png" />
<div class="section" id="data-layout">
<h3>数据布局<a class="headerlink" href="#data-layout" title="永久链接至标题">¶</a></h3>
<p>We describe the placeholder tensors <code class="code docutils literal notranslate"><span class="pre">A</span></code>, and <code class="code docutils literal notranslate"><span class="pre">B</span></code> in a tiled data
format to match the data layout requirements imposed by the VTA tensor core.</p>
<div class="admonition note">
<p class="admonition-title">注解</p>
<p><strong>Data Tiling</strong></p>
<p>One source of complexity when targeting accelerators is to make sure
that the data layout matches the layout imposed by the accelerator design.
VTA is designed around a <em>tensor core</em> that performs, one matrix-matrix
operation per cycle between an activation matrix and a weight matrix,
adding the result matrix to an accumulator matrix, as shown in the
figure below.</p>
<a class="reference internal image-reference" href="https://raw.githubusercontent.com/uwsampl/web-data/main/vta/tutorial/tensor_core.png"><img alt="https://raw.githubusercontent.com/uwsampl/web-data/main/vta/tutorial/tensor_core.png" class="align-center" src="https://raw.githubusercontent.com/uwsampl/web-data/main/vta/tutorial/tensor_core.png" style="width: 480px;" /></a>
<p>The dimensions of that matrix-matrix multiplication are specified in
the <code class="code docutils literal notranslate"><span class="pre">vta_config.json</span></code> configuration file.
The activation matrix has a <code class="code docutils literal notranslate"><span class="pre">(BATCH,</span> <span class="pre">BLOCK_IN)</span></code> shape
and the transposed weight matrix has a <code class="code docutils literal notranslate"><span class="pre">(BLOCK_OUT,</span> <span class="pre">BLOCK_IN)</span></code> shape,
thus inferring that the resulting output matrix has a
<code class="code docutils literal notranslate"><span class="pre">(BATCH,</span> <span class="pre">BLOCK_OUT)</span></code> shape.
Consequently input and output tensors processed by VTA need to be
tiled according to these aforementioned dimension.</p>
<p>The diagram below shows the impact of data tiling on a matrix that is
originally of shape (4, 8).
Tiling by a (2, 2) tile shape ensures that data within each tile is
contiguous.
The resulting tiled tensor has a shape of (2, 4, 2, 2).</p>
<a class="reference internal image-reference" href="https://raw.githubusercontent.com/uwsampl/web-data/main/vta/tutorial/data_tiling.png"><img alt="https://raw.githubusercontent.com/uwsampl/web-data/main/vta/tutorial/data_tiling.png" class="align-center" src="https://raw.githubusercontent.com/uwsampl/web-data/main/vta/tutorial/data_tiling.png" style="width: 480px;" /></a>
</div>
<p>We first define the variables <code class="code docutils literal notranslate"><span class="pre">m</span></code>, <code class="code docutils literal notranslate"><span class="pre">n</span></code>, <code class="code docutils literal notranslate"><span class="pre">o</span></code> to represent
the shape of the matrix multiplication. These variables are multiplicative
factors over the <code class="code docutils literal notranslate"><span class="pre">BLOCK_OUT</span></code>, <code class="code docutils literal notranslate"><span class="pre">BLOCK_IN</span></code>, and <code class="code docutils literal notranslate"><span class="pre">BATCH</span></code>
tensor dimensions respectively.
By default, the configuration file sets <code class="code docutils literal notranslate"><span class="pre">BATCH</span></code>, <code class="code docutils literal notranslate"><span class="pre">BLOCK_IN</span></code>, and
<code class="code docutils literal notranslate"><span class="pre">BLOCK_OUT</span></code> to be 1, 16 and 16 respectively (<code class="code docutils literal notranslate"><span class="pre">BATCH</span></code> being set to
1 implies that our compute building block is vector-matrix multiply).</p>
<div class="admonition note">
<p class="admonition-title">注解</p>
<p><strong>Data Types</strong></p>
<p>It’s important to not only match the inner-tile
dimension of VTA’s tensor core, but also to match the specific data types
expected by VTA.
VTA for now only supports fixed point data types, which integer width is
specified in the <code class="code docutils literal notranslate"><span class="pre">vta_config.json</span></code> file by <code class="code docutils literal notranslate"><span class="pre">INP_WIDTH</span></code> and
<code class="code docutils literal notranslate"><span class="pre">WGT_WIDTH</span></code> for the activations and weights data types respectively.
In addition, the accumulator data type integer width is specified by
<code class="code docutils literal notranslate"><span class="pre">ACC_WIDTH</span></code>.</p>
</div>
<p>By default, the configuration file sets <code class="code docutils literal notranslate"><span class="pre">INP_WIDTH</span></code>
and <code class="code docutils literal notranslate"><span class="pre">WGT_WIDTH</span></code> to 8.
The accumulator width <code class="code docutils literal notranslate"><span class="pre">ACC_WIDTH</span></code> is set to 32, in order to avoid
overflow during accumulation.
As a result, <code class="code docutils literal notranslate"><span class="pre">env.inp_dtype</span></code> and <code class="code docutils literal notranslate"><span class="pre">env.wgt_dtype</span></code> are all
narrow 8-bit integers, while <code class="code docutils literal notranslate"><span class="pre">env.acc_dtype</span></code> is a standard 32-bit
integer.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># Output channel factor m - total 16x16=256 output channels</span>
<span class="n">m</span> <span class="o">=</span> <span class="mi">16</span>
<span class="c1"># Input channel factor n - total 16x16=256 input channels</span>
<span class="n">n</span> <span class="o">=</span> <span class="mi">16</span>
<span class="c1"># Batch factor o (we use single batch inference)</span>
<span class="n">o</span> <span class="o">=</span> <span class="mi">1</span>
<span class="c1"># A placeholder tensor in tiled data format</span>
<span class="n">A</span> <span class="o">=</span> <span class="n">te</span><span class="o">.</span><span class="n">placeholder</span><span class="p">((</span><span class="n">o</span><span class="p">,</span> <span class="n">n</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BATCH</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BLOCK_IN</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;A&quot;</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">env</span><span class="o">.</span><span class="n">inp_dtype</span><span class="p">)</span>
<span class="c1"># B placeholder tensor in tiled data format</span>
<span class="n">B</span> <span class="o">=</span> <span class="n">te</span><span class="o">.</span><span class="n">placeholder</span><span class="p">((</span><span class="n">m</span><span class="p">,</span> <span class="n">n</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BLOCK_OUT</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BLOCK_IN</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;B&quot;</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">env</span><span class="o">.</span><span class="n">wgt_dtype</span><span class="p">)</span>
<span class="c1"># A copy buffer</span>
<span class="n">A_buf</span> <span class="o">=</span> <span class="n">te</span><span class="o">.</span><span class="n">compute</span><span class="p">((</span><span class="n">o</span><span class="p">,</span> <span class="n">n</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BATCH</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BLOCK_IN</span><span class="p">),</span> <span class="k">lambda</span> <span class="o">*</span><span class="n">i</span><span class="p">:</span> <span class="n">A</span><span class="p">(</span><span class="o">*</span><span class="n">i</span><span class="p">),</span> <span class="s2">&quot;A_buf&quot;</span><span class="p">)</span>
<span class="c1"># B copy buffer</span>
<span class="n">B_buf</span> <span class="o">=</span> <span class="n">te</span><span class="o">.</span><span class="n">compute</span><span class="p">((</span><span class="n">m</span><span class="p">,</span> <span class="n">n</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BLOCK_OUT</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BLOCK_IN</span><span class="p">),</span> <span class="k">lambda</span> <span class="o">*</span><span class="n">i</span><span class="p">:</span> <span class="n">B</span><span class="p">(</span><span class="o">*</span><span class="n">i</span><span class="p">),</span> <span class="s2">&quot;B_buf&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="section" id="matrix-multiplication">
<h3>Matrix Multiplication<a class="headerlink" href="#matrix-multiplication" title="永久链接至标题">¶</a></h3>
<p>Now we’re ready to describe the matrix multiplication result tensor <code class="code docutils literal notranslate"><span class="pre">C</span></code>,
with another compute operation.
The compute function takes the shape of the tensor, as well as a lambda
function that describes the computation rule for each position of the tensor.</p>
<p>In order to implement matrix multiplication, the lambda function needs to
include a reduction formula over the input channel dimension axes.
To create a reduction formula, we can declare a reduction axis using
<code class="code docutils literal notranslate"><span class="pre">te.reduce_axis</span></code>, which takes in the range of reductions.
<code class="code docutils literal notranslate"><span class="pre">te.sum</span></code> takes in the expression to be reduced as well as
the reduction axes to compute the sum of value over all k in the declared
ranges.</p>
<p>Note that the reduction needs to be performed over 32-bit <code class="code docutils literal notranslate"><span class="pre">env.acc_dtype</span></code>
accumulator data types.</p>
<p>No computation happens during this phase, as we are only declaring how
the computation should be done.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># Outer input feature reduction axis</span>
<span class="n">ko</span> <span class="o">=</span> <span class="n">te</span><span class="o">.</span><span class="n">reduce_axis</span><span class="p">((</span><span class="mi">0</span><span class="p">,</span> <span class="n">n</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;ko&quot;</span><span class="p">)</span>
<span class="c1"># Inner input feature reduction axis</span>
<span class="n">ki</span> <span class="o">=</span> <span class="n">te</span><span class="o">.</span><span class="n">reduce_axis</span><span class="p">((</span><span class="mi">0</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BLOCK_IN</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;ki&quot;</span><span class="p">)</span>
<span class="c1"># Describe the in-VTA matrix multiplication</span>
<span class="n">C_buf</span> <span class="o">=</span> <span class="n">te</span><span class="o">.</span><span class="n">compute</span><span class="p">(</span>
    <span class="p">(</span><span class="n">o</span><span class="p">,</span> <span class="n">m</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BATCH</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BLOCK_OUT</span><span class="p">),</span>
    <span class="k">lambda</span> <span class="n">bo</span><span class="p">,</span> <span class="n">co</span><span class="p">,</span> <span class="n">bi</span><span class="p">,</span> <span class="n">ci</span><span class="p">:</span> <span class="n">te</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span>
        <span class="n">A_buf</span><span class="p">[</span><span class="n">bo</span><span class="p">,</span> <span class="n">ko</span><span class="p">,</span> <span class="n">bi</span><span class="p">,</span> <span class="n">ki</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">env</span><span class="o">.</span><span class="n">acc_dtype</span><span class="p">)</span> <span class="o">*</span> <span class="n">B_buf</span><span class="p">[</span><span class="n">co</span><span class="p">,</span> <span class="n">ko</span><span class="p">,</span> <span class="n">ci</span><span class="p">,</span> <span class="n">ki</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">env</span><span class="o">.</span><span class="n">acc_dtype</span><span class="p">),</span>
        <span class="n">axis</span><span class="o">=</span><span class="p">[</span><span class="n">ko</span><span class="p">,</span> <span class="n">ki</span><span class="p">],</span>
    <span class="p">),</span>
    <span class="n">name</span><span class="o">=</span><span class="s2">&quot;C_buf&quot;</span><span class="p">,</span>
<span class="p">)</span>
</pre></div>
</div>
</div>
<div class="section" id="casting-the-results">
<h3>Casting the Results<a class="headerlink" href="#casting-the-results" title="永久链接至标题">¶</a></h3>
<p>After the computation is done, we’ll need to send the results computed by VTA
back to main memory.</p>
<div class="admonition note">
<p class="admonition-title">注解</p>
<p><strong>Memory Store Restrictions</strong></p>
<p>One specificity of VTA is that it only supports DRAM stores in the narrow
<code class="code docutils literal notranslate"><span class="pre">env.inp_dtype</span></code> data type format.
This lets us reduce the data footprint for memory transfers, but also lets
us quantize the wide accumulator data type down to a data format that
matches the input activation data type.
This means that in the context of neural network inference, the outputs
of a given layer after activation can be consumed directly by the next
layer.</p>
</div>
<p>We perform one last typecast operation to the narrow
input activation data format.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># Cast to output type, and send to main memory</span>
<span class="n">C</span> <span class="o">=</span> <span class="n">te</span><span class="o">.</span><span class="n">compute</span><span class="p">(</span>
    <span class="p">(</span><span class="n">o</span><span class="p">,</span> <span class="n">m</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BATCH</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BLOCK_OUT</span><span class="p">),</span> <span class="k">lambda</span> <span class="o">*</span><span class="n">i</span><span class="p">:</span> <span class="n">C_buf</span><span class="p">(</span><span class="o">*</span><span class="n">i</span><span class="p">)</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">env</span><span class="o">.</span><span class="n">inp_dtype</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;C&quot;</span>
<span class="p">)</span>
</pre></div>
</div>
<p>This concludes the computation declaration part of this tutorial.</p>
</div>
</div>
<div class="section" id="scheduling-the-computation">
<h2>Scheduling the Computation<a class="headerlink" href="#scheduling-the-computation" title="永久链接至标题">¶</a></h2>
<p>While the above lines describes the computation rule, we can obtain
<code class="code docutils literal notranslate"><span class="pre">C</span></code> in many ways.
TVM asks the user to provide an implementation of the computation called
<em>schedule</em>.</p>
<p>A schedule is a set of transformations to an original computation that
transforms the implementation of the computation without affecting
correctness.
This simple VTA programming tutorial aims to demonstrate basic schedule
transformations that will map the original schedule down to VTA hardware
primitives.</p>
<div class="section" id="default-schedule">
<h3>Default Schedule<a class="headerlink" href="#default-schedule" title="永久链接至标题">¶</a></h3>
<p>After we construct the schedule, by default the schedule computes
<code class="code docutils literal notranslate"><span class="pre">C</span></code> in the following way:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># Let&#39;s take a look at the generated schedule</span>
<span class="n">s</span> <span class="o">=</span> <span class="n">te</span><span class="o">.</span><span class="n">create_schedule</span><span class="p">(</span><span class="n">C</span><span class="o">.</span><span class="n">op</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">tvm</span><span class="o">.</span><span class="n">lower</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="p">[</span><span class="n">A</span><span class="p">,</span> <span class="n">B</span><span class="p">,</span> <span class="n">C</span><span class="p">],</span> <span class="n">simple_mode</span><span class="o">=</span><span class="kc">True</span><span class="p">))</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">输出:</p>
<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>primfn(A_1: handle, B_1: handle, C_1: handle) -&gt; ()
  attr = {&quot;from_legacy_te_schedule&quot;: True, &quot;global_symbol&quot;: &quot;main&quot;, &quot;tir.noalias&quot;: True}
  buffers = {C: Buffer(C_2: Pointer(int8), int8, [1, 16, 1, 16], []),
             A: Buffer(A_2: Pointer(int8), int8, [1, 16, 1, 16], []),
             B: Buffer(B_2: Pointer(int8), int8, [16, 16, 16, 16], [])}
  buffer_map = {A_1: A, B_1: B, C_1: C} {
  allocate(A_buf: Pointer(global int8), int8, [256]), storage_scope = global;
  allocate(B_buf: Pointer(global int8), int8, [65536]), storage_scope = global;
  allocate(C_buf: Pointer(global int32), int32, [256]), storage_scope = global {
    for (i1: int32, 0, 16) {
      for (i3: int32, 0, 16) {
        A_buf[((i1*16) + i3)] = (int8*)A_2[((i1*16) + i3)]
      }
    }
    for (i0: int32, 0, 16) {
      for (i1_1: int32, 0, 16) {
        for (i2: int32, 0, 16) {
          for (i3_1: int32, 0, 16) {
            B_buf[((((i0*4096) + (i1_1*256)) + (i2*16)) + i3_1)] = (int8*)B_2[((((i0*4096) + (i1_1*256)) + (i2*16)) + i3_1)]
          }
        }
      }
    }
    for (co: int32, 0, 16) {
      for (ci: int32, 0, 16) {
        C_buf[((co*16) + ci)] = 0
        for (ko: int32, 0, 16) {
          for (ki: int32, 0, 16) {
            C_buf[((co*16) + ci)] = ((int32*)C_buf[((co*16) + ci)] + (cast(int32, (int8*)A_buf[((ko*16) + ki)])*cast(int32, (int8*)B_buf[((((co*4096) + (ko*256)) + (ci*16)) + ki)])))
          }
        }
      }
    }
    for (i1_2: int32, 0, 16) {
      for (i3_2: int32, 0, 16) {
        C_2[((i1_2*16) + i3_2)] = cast(int8, (int32*)C_buf[((i1_2*16) + i3_2)])
      }
    }
  }
}
</pre></div>
</div>
<p>Although this schedule makes sense, it won’t compile to VTA.
In order to obtain correct code generation, we need to apply scheduling
primitives and code annotation that will transform the schedule into
one that can be directly lowered onto VTA hardware intrinsics.
Those include:</p>
<blockquote>
<div><ul class="simple">
<li><p>DMA copy operations which will take globally-scoped tensors and copy
those into locally-scoped tensors.</p></li>
<li><p>Tensor operations that will perform the matrix multiplication.</p></li>
</ul>
</div></blockquote>
</div>
<div class="section" id="buffer-scopes">
<h3>Buffer Scopes<a class="headerlink" href="#buffer-scopes" title="永久链接至标题">¶</a></h3>
<p>First, we set the scope of the buffers to tell TVM that these buffers
will be living in the VTA’s on-chip SRAM caches.
Below, we tell TVM that <code class="code docutils literal notranslate"><span class="pre">A_buf</span></code>, <code class="code docutils literal notranslate"><span class="pre">B_buf</span></code>, <code class="code docutils literal notranslate"><span class="pre">C_buf</span></code>
will respectively live in VTA’s on-chip input, weight and accumulator
memory.</p>
<div class="admonition note">
<p class="admonition-title">注解</p>
<p><strong>VTA’s On-Chip SRAMs</strong></p>
<p>VTA has three different memory scopes, each corresponding to different
on-chip SRAM buffers.</p>
<blockquote>
<div><ul class="simple">
<li><p><code class="code docutils literal notranslate"><span class="pre">env.inp_scope</span></code>: Input buffer, which is a read-only SRAM buffer
that stores input matrices of shape <code class="code docutils literal notranslate"><span class="pre">(env.BATCH,</span> <span class="pre">env.BLOCK_IN)</span></code>
of type <code class="code docutils literal notranslate"><span class="pre">env.inp_dtype</span></code>. The input buffer contains
<cite>2 ^ LOG_INP_BUFF_SIZE</cite> matrix elements (as specified in the
<code class="code docutils literal notranslate"><span class="pre">vta_config.json</span></code> file).</p></li>
<li><p><code class="code docutils literal notranslate"><span class="pre">env.wgt_scope</span></code>: Weight buffer, which is a read-only SRAM buffer
that stores weight matrices of shape <code class="code docutils literal notranslate"><span class="pre">(env.BLOCK_OUT,</span> <span class="pre">env.BLOCK_IN)</span></code>
of type <code class="code docutils literal notranslate"><span class="pre">env.wgt_dtype</span></code>. The weight buffer contains
<cite>2 ^ LOG_WGT_BUFF_SIZE</cite> matrix elements.</p></li>
<li><p><code class="code docutils literal notranslate"><span class="pre">env.acc_scope</span></code>: Accumulator buffer, which is a read/write SRAM
buffer that stores accumulator matrices of shape
<code class="code docutils literal notranslate"><span class="pre">(env.BATCH,</span> <span class="pre">env.BLOCK_OUT)</span></code> of type <code class="code docutils literal notranslate"><span class="pre">env.acc_dtype</span></code>.
The accumulator buffer is VTA’s general purpose register file: it holds
both intermediate results of convolutions and matrix multiplications
as well as intermediate results of pooling, batch normalization, and
activation layers. The accumulator buffer contains
<cite>2 ^ LOG_ACC_BUFF_SIZE</cite> matrix elements.</p></li>
</ul>
</div></blockquote>
</div>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># Set the intermediate tensor&#39;s scope to VTA&#39;s on-chip buffers</span>
<span class="n">s</span><span class="p">[</span><span class="n">A_buf</span><span class="p">]</span><span class="o">.</span><span class="n">set_scope</span><span class="p">(</span><span class="n">env</span><span class="o">.</span><span class="n">inp_scope</span><span class="p">)</span>
<span class="n">s</span><span class="p">[</span><span class="n">B_buf</span><span class="p">]</span><span class="o">.</span><span class="n">set_scope</span><span class="p">(</span><span class="n">env</span><span class="o">.</span><span class="n">wgt_scope</span><span class="p">)</span>
<span class="n">s</span><span class="p">[</span><span class="n">C_buf</span><span class="p">]</span><span class="o">.</span><span class="n">set_scope</span><span class="p">(</span><span class="n">env</span><span class="o">.</span><span class="n">acc_scope</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="section" id="dma-transfers">
<h3>DMA Transfers<a class="headerlink" href="#dma-transfers" title="永久链接至标题">¶</a></h3>
<p>We need to schedule DMA transfers to move data living in DRAM to
and from the VTA on-chip buffers.
This can be achieved using the <code class="code docutils literal notranslate"><span class="pre">compute_at</span></code> schedule primitive
which nests the copying of the buffers into the computation loop
that performs the matrix multiplication.</p>
<p>We insert <code class="code docutils literal notranslate"><span class="pre">dma_copy</span></code> pragmas to indicate to the compiler
that the copy operations will be performed in bulk via DMA,
which is common in hardware accelerators.
Finally, we print the temporary schedule to observe the effects of
moving the copy operations into the matrix multiplication loop.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># Move buffer copy into matrix multiply loop</span>
<span class="n">s</span><span class="p">[</span><span class="n">A_buf</span><span class="p">]</span><span class="o">.</span><span class="n">compute_at</span><span class="p">(</span><span class="n">s</span><span class="p">[</span><span class="n">C_buf</span><span class="p">],</span> <span class="n">ko</span><span class="p">)</span>
<span class="n">s</span><span class="p">[</span><span class="n">B_buf</span><span class="p">]</span><span class="o">.</span><span class="n">compute_at</span><span class="p">(</span><span class="n">s</span><span class="p">[</span><span class="n">C_buf</span><span class="p">],</span> <span class="n">ko</span><span class="p">)</span>

<span class="c1"># Tag the buffer copies with the DMA pragma to insert a DMA transfer</span>
<span class="n">s</span><span class="p">[</span><span class="n">A_buf</span><span class="p">]</span><span class="o">.</span><span class="n">pragma</span><span class="p">(</span><span class="n">s</span><span class="p">[</span><span class="n">A_buf</span><span class="p">]</span><span class="o">.</span><span class="n">op</span><span class="o">.</span><span class="n">axis</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">env</span><span class="o">.</span><span class="n">dma_copy</span><span class="p">)</span>
<span class="n">s</span><span class="p">[</span><span class="n">B_buf</span><span class="p">]</span><span class="o">.</span><span class="n">pragma</span><span class="p">(</span><span class="n">s</span><span class="p">[</span><span class="n">B_buf</span><span class="p">]</span><span class="o">.</span><span class="n">op</span><span class="o">.</span><span class="n">axis</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">env</span><span class="o">.</span><span class="n">dma_copy</span><span class="p">)</span>
<span class="n">s</span><span class="p">[</span><span class="n">C</span><span class="p">]</span><span class="o">.</span><span class="n">pragma</span><span class="p">(</span><span class="n">s</span><span class="p">[</span><span class="n">C</span><span class="p">]</span><span class="o">.</span><span class="n">op</span><span class="o">.</span><span class="n">axis</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">env</span><span class="o">.</span><span class="n">dma_copy</span><span class="p">)</span>

<span class="c1"># Let&#39;s take a look at the transformed schedule</span>
<span class="nb">print</span><span class="p">(</span><span class="n">tvm</span><span class="o">.</span><span class="n">lower</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="p">[</span><span class="n">A</span><span class="p">,</span> <span class="n">B</span><span class="p">,</span> <span class="n">C</span><span class="p">],</span> <span class="n">simple_mode</span><span class="o">=</span><span class="kc">True</span><span class="p">))</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">输出:</p>
<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>primfn(A_1: handle, B_1: handle, C_1: handle) -&gt; ()
  attr = {&quot;from_legacy_te_schedule&quot;: True, &quot;global_symbol&quot;: &quot;main&quot;, &quot;tir.noalias&quot;: True}
  buffers = {C: Buffer(C_2: Pointer(int8), int8, [1, 16, 1, 16], []),
             A: Buffer(A_2: Pointer(int8), int8, [1, 16, 1, 16], []),
             B: Buffer(B_2: Pointer(int8), int8, [16, 16, 16, 16], [])}
  buffer_map = {A_1: A, B_1: B, C_1: C} {
  allocate(C_buf: Pointer(local.acc_buffer int32), int32, [256]), storage_scope = local.acc_buffer;
  allocate(A_buf: Pointer(local.inp_buffer int8), int8, [16]), storage_scope = local.inp_buffer;
  allocate(B_buf: Pointer(local.wgt_buffer int8), int8, [16]), storage_scope = local.wgt_buffer {
    for (co: int32, 0, 16) {
      for (ci: int32, 0, 16) {
        C_buf[((co*16) + ci)] = 0
        for (ko: int32, 0, 16) {
          attr [IterVar(i0: int32, (nullptr), &quot;DataPar&quot;, &quot;&quot;)] &quot;pragma_dma_copy&quot; = 1;
          for (i3: int32, 0, 16) {
            A_buf[i3] = (int8*)A_2[((ko*16) + i3)]
          }
          attr [IterVar(i0_1: int32, (nullptr), &quot;DataPar&quot;, &quot;&quot;)] &quot;pragma_dma_copy&quot; = 1;
          for (i3_1: int32, 0, 16) {
            B_buf[i3_1] = (int8*)B_2[((((co*4096) + (ko*256)) + (ci*16)) + i3_1)]
          }
          for (ki: int32, 0, 16) {
            C_buf[((co*16) + ci)] = ((int32*)C_buf[((co*16) + ci)] + (cast(int32, (int8*)A_buf[ki])*cast(int32, (int8*)B_buf[ki])))
          }
        }
      }
    }
    attr [IterVar(i0_2: int32, (nullptr), &quot;DataPar&quot;, &quot;&quot;)] &quot;pragma_dma_copy&quot; = 1;
    for (i1: int32, 0, 16) {
      for (i3_2: int32, 0, 16) {
        C_2[((i1*16) + i3_2)] = cast(int8, (int32*)C_buf[((i1*16) + i3_2)])
      }
    }
  }
}
</pre></div>
</div>
</div>
<div class="section" id="tensorization">
<h3>Tensorization<a class="headerlink" href="#tensorization" title="永久链接至标题">¶</a></h3>
<p>The last step of the schedule transformation consists in applying
<em>tensorization</em> to our schedule.
Tensorization is analogous to vectorization, but extends the concept
to a higher-dimensional unit of computation.
Consequently, tensorization imposes data layout constraints as discussed
earlier when declaring the data layout input placeholders.
We’ve already arranged our tensors in a tiled format, so the next thing
we need to perform is loop reordering to accommodate for tensorization.</p>
<p>Here we choose to move the outermost reduction axis all the way out.
This dictates that we first iterate over input channels, then batch
dimensions, and finally output channels.
Lastly, we apply the tensorization scheduling primitive <code class="code docutils literal notranslate"><span class="pre">tensorize</span></code>
along the outer axis of the inner-most matrix matrix multiplication tensor
block.
We print the finalized schedule that is ready for code-generation
by the VTA runtime JIT compiler.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">s</span><span class="p">[</span><span class="n">C_buf</span><span class="p">]</span><span class="o">.</span><span class="n">reorder</span><span class="p">(</span>
    <span class="n">ko</span><span class="p">,</span> <span class="n">s</span><span class="p">[</span><span class="n">C_buf</span><span class="p">]</span><span class="o">.</span><span class="n">op</span><span class="o">.</span><span class="n">axis</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">s</span><span class="p">[</span><span class="n">C_buf</span><span class="p">]</span><span class="o">.</span><span class="n">op</span><span class="o">.</span><span class="n">axis</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">s</span><span class="p">[</span><span class="n">C_buf</span><span class="p">]</span><span class="o">.</span><span class="n">op</span><span class="o">.</span><span class="n">axis</span><span class="p">[</span><span class="mi">2</span><span class="p">],</span> <span class="n">s</span><span class="p">[</span><span class="n">C_buf</span><span class="p">]</span><span class="o">.</span><span class="n">op</span><span class="o">.</span><span class="n">axis</span><span class="p">[</span><span class="mi">3</span><span class="p">],</span> <span class="n">ki</span>
<span class="p">)</span>
<span class="n">s</span><span class="p">[</span><span class="n">C_buf</span><span class="p">]</span><span class="o">.</span><span class="n">tensorize</span><span class="p">(</span><span class="n">s</span><span class="p">[</span><span class="n">C_buf</span><span class="p">]</span><span class="o">.</span><span class="n">op</span><span class="o">.</span><span class="n">axis</span><span class="p">[</span><span class="mi">2</span><span class="p">],</span> <span class="n">env</span><span class="o">.</span><span class="n">gemm</span><span class="p">)</span>

<span class="c1"># Let&#39;s take a look at the finalized schedule</span>
<span class="nb">print</span><span class="p">(</span><span class="n">vta</span><span class="o">.</span><span class="n">lower</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="p">[</span><span class="n">A</span><span class="p">,</span> <span class="n">B</span><span class="p">,</span> <span class="n">C</span><span class="p">],</span> <span class="n">simple_mode</span><span class="o">=</span><span class="kc">True</span><span class="p">))</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">输出:</p>
<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>primfn(A_1: handle, B_1: handle, C_1: handle) -&gt; ()
  attr = {&quot;from_legacy_te_schedule&quot;: True, &quot;global_symbol&quot;: &quot;main&quot;, &quot;tir.noalias&quot;: True}
  buffers = {C: Buffer(C_2: Pointer(int8), int8, [1, 16, 1, 16], []),
             A: Buffer(A_2: Pointer(int8), int8, [1, 16, 1, 16], []),
             B: Buffer(B_2: Pointer(int8), int8, [16, 16, 16, 16], [])}
  buffer_map = {A_1: A, B_1: B, C_1: C} {
  attr [IterVar(vta: int32, (nullptr), &quot;ThreadIndex&quot;, &quot;vta&quot;)] &quot;coproc_scope&quot; = 2 {
    attr [IterVar(vta, (nullptr), &quot;ThreadIndex&quot;, &quot;vta&quot;)] &quot;coproc_uop_scope&quot; = &quot;VTAPushGEMMOp&quot; {
      @tir.call_extern(&quot;VTAUopLoopBegin&quot;, 16, 1, 0, 0, dtype=int32)
      @tir.vta.uop_push(0, 1, 0, 0, 0, 0, 0, 0, dtype=int32)
      @tir.call_extern(&quot;VTAUopLoopEnd&quot;, dtype=int32)
    }
    @tir.vta.coproc_dep_push(2, 1, dtype=int32)
  }
  for (ko: int32, 0, 16) {
    attr [IterVar(vta, (nullptr), &quot;ThreadIndex&quot;, &quot;vta&quot;)] &quot;coproc_scope&quot; = 1 {
      @tir.vta.coproc_dep_pop(2, 1, dtype=int32)
      @tir.call_extern(&quot;VTALoadBuffer2D&quot;, @tir.tvm_thread_context(@tir.vta.command_handle(, dtype=handle), dtype=handle), A_2, ko, 1, 1, 1, 0, 0, 0, 0, 0, 2, dtype=int32)
      @tir.call_extern(&quot;VTALoadBuffer2D&quot;, @tir.tvm_thread_context(@tir.vta.command_handle(, dtype=handle), dtype=handle), B_2, ko, 1, 16, 16, 0, 0, 0, 0, 0, 1, dtype=int32)
      @tir.vta.coproc_dep_push(1, 2, dtype=int32)
    }
    attr [IterVar(vta, (nullptr), &quot;ThreadIndex&quot;, &quot;vta&quot;)] &quot;coproc_scope&quot; = 2 {
      @tir.vta.coproc_dep_pop(1, 2, dtype=int32)
      attr [IterVar(vta, (nullptr), &quot;ThreadIndex&quot;, &quot;vta&quot;)] &quot;coproc_uop_scope&quot; = &quot;VTAPushGEMMOp&quot; {
        @tir.call_extern(&quot;VTAUopLoopBegin&quot;, 16, 1, 0, 1, dtype=int32)
        @tir.vta.uop_push(0, 0, 0, 0, 0, 0, 0, 0, dtype=int32)
        @tir.call_extern(&quot;VTAUopLoopEnd&quot;, dtype=int32)
      }
      @tir.vta.coproc_dep_push(2, 1, dtype=int32)
    }
  }
  @tir.vta.coproc_dep_push(2, 3, dtype=int32)
  @tir.vta.coproc_dep_pop(2, 1, dtype=int32)
  attr [IterVar(vta, (nullptr), &quot;ThreadIndex&quot;, &quot;vta&quot;)] &quot;coproc_scope&quot; = 3 {
    @tir.vta.coproc_dep_pop(2, 3, dtype=int32)
    @tir.call_extern(&quot;VTAStoreBuffer2D&quot;, @tir.tvm_thread_context(@tir.vta.command_handle(, dtype=handle), dtype=handle), 0, 4, C_2, 0, 16, 1, 16, dtype=int32)
  }
  @tir.vta.coproc_sync(, dtype=int32)
}
</pre></div>
</div>
<p>This concludes the scheduling portion of this tutorial.</p>
</div>
</div>
<div class="section" id="tvm-compilation">
<h2>TVM Compilation<a class="headerlink" href="#tvm-compilation" title="永久链接至标题">¶</a></h2>
<p>After we have finished specifying the schedule, we can compile it
into a TVM function.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># Build GEMM VTA kernel</span>
<span class="n">my_gemm</span> <span class="o">=</span> <span class="n">vta</span><span class="o">.</span><span class="n">build</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="p">[</span><span class="n">A</span><span class="p">,</span> <span class="n">B</span><span class="p">,</span> <span class="n">C</span><span class="p">],</span> <span class="s2">&quot;ext_dev&quot;</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">target_host</span><span class="p">,</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;my_gemm&quot;</span><span class="p">)</span>

<span class="c1"># Write the compiled module into an object file.</span>
<span class="n">temp</span> <span class="o">=</span> <span class="n">utils</span><span class="o">.</span><span class="n">tempdir</span><span class="p">()</span>
<span class="n">my_gemm</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">temp</span><span class="o">.</span><span class="n">relpath</span><span class="p">(</span><span class="s2">&quot;gemm.o&quot;</span><span class="p">))</span>

<span class="c1"># Send the executable over RPC</span>
<span class="n">remote</span><span class="o">.</span><span class="n">upload</span><span class="p">(</span><span class="n">temp</span><span class="o">.</span><span class="n">relpath</span><span class="p">(</span><span class="s2">&quot;gemm.o&quot;</span><span class="p">))</span>

<span class="c1"># Load the compiled module</span>
<span class="n">f</span> <span class="o">=</span> <span class="n">remote</span><span class="o">.</span><span class="n">load_module</span><span class="p">(</span><span class="s2">&quot;gemm.o&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="section" id="running-the-function">
<h2>Running the Function<a class="headerlink" href="#running-the-function" title="永久链接至标题">¶</a></h2>
<p>The compiled TVM function uses a concise C API and can be invoked from
code language.</p>
<p>TVM provides an array API in python to aid quick testing and prototyping.
The array API is based on <a class="reference external" href="https://github.com/dmlc/dlpack">DLPack</a> standard.</p>
<ul class="simple">
<li><p>We first create a remote context (for remote execution on the Pynq).</p></li>
<li><p>Then <code class="code docutils literal notranslate"><span class="pre">tvm.nd.array</span></code> formats the data accordingly.</p></li>
<li><p><code class="code docutils literal notranslate"><span class="pre">f()</span></code> runs the actual computation.</p></li>
<li><p><code class="code docutils literal notranslate"><span class="pre">numpy()</span></code> copies the result array back in a format that can be
interpreted.</p></li>
</ul>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># Get the remote device context</span>
<span class="n">ctx</span> <span class="o">=</span> <span class="n">remote</span><span class="o">.</span><span class="n">ext_dev</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="c1"># Initialize the A and B arrays randomly in the int range of (-128, 128]</span>
<span class="n">A_orig</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="o">-</span><span class="mi">128</span><span class="p">,</span> <span class="mi">128</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="p">(</span><span class="n">o</span> <span class="o">*</span> <span class="n">env</span><span class="o">.</span><span class="n">BATCH</span><span class="p">,</span> <span class="n">n</span> <span class="o">*</span> <span class="n">env</span><span class="o">.</span><span class="n">BLOCK_IN</span><span class="p">))</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">A</span><span class="o">.</span><span class="n">dtype</span><span class="p">)</span>
<span class="n">B_orig</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="o">-</span><span class="mi">128</span><span class="p">,</span> <span class="mi">128</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="p">(</span><span class="n">m</span> <span class="o">*</span> <span class="n">env</span><span class="o">.</span><span class="n">BLOCK_OUT</span><span class="p">,</span> <span class="n">n</span> <span class="o">*</span> <span class="n">env</span><span class="o">.</span><span class="n">BLOCK_IN</span><span class="p">))</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">B</span><span class="o">.</span><span class="n">dtype</span><span class="p">)</span>

<span class="c1"># Apply packing to the A and B arrays from a 2D to a 4D packed layout</span>
<span class="n">A_packed</span> <span class="o">=</span> <span class="n">A_orig</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="n">o</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BATCH</span><span class="p">,</span> <span class="n">n</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BLOCK_IN</span><span class="p">)</span><span class="o">.</span><span class="n">transpose</span><span class="p">((</span><span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">))</span>
<span class="n">B_packed</span> <span class="o">=</span> <span class="n">B_orig</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="n">m</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BLOCK_OUT</span><span class="p">,</span> <span class="n">n</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BLOCK_IN</span><span class="p">)</span><span class="o">.</span><span class="n">transpose</span><span class="p">((</span><span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">))</span>

<span class="c1"># Format the input/output arrays with tvm.nd.array to the DLPack standard</span>
<span class="n">A_nd</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">nd</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">A_packed</span><span class="p">,</span> <span class="n">ctx</span><span class="p">)</span>
<span class="n">B_nd</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">nd</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">B_packed</span><span class="p">,</span> <span class="n">ctx</span><span class="p">)</span>
<span class="n">C_nd</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">nd</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="n">o</span><span class="p">,</span> <span class="n">m</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BATCH</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BLOCK_OUT</span><span class="p">))</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">C</span><span class="o">.</span><span class="n">dtype</span><span class="p">),</span> <span class="n">ctx</span><span class="p">)</span>

<span class="c1"># Clear stats</span>
<span class="k">if</span> <span class="n">env</span><span class="o">.</span><span class="n">TARGET</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;sim&quot;</span><span class="p">,</span> <span class="s2">&quot;tsim&quot;</span><span class="p">]:</span>
    <span class="n">simulator</span><span class="o">.</span><span class="n">clear_stats</span><span class="p">()</span>

<span class="c1"># Invoke the module to perform the computation</span>
<span class="n">f</span><span class="p">(</span><span class="n">A_nd</span><span class="p">,</span> <span class="n">B_nd</span><span class="p">,</span> <span class="n">C_nd</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="section" id="verifying-correctness">
<h2>Verifying Correctness<a class="headerlink" href="#verifying-correctness" title="永久链接至标题">¶</a></h2>
<p>Compute the reference result with numpy and assert that the output of the
matrix multiplication indeed is correct</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># Compute reference result with numpy</span>
<span class="n">C_ref</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">A_orig</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">env</span><span class="o">.</span><span class="n">acc_dtype</span><span class="p">),</span> <span class="n">B_orig</span><span class="o">.</span><span class="n">T</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">env</span><span class="o">.</span><span class="n">acc_dtype</span><span class="p">))</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">C</span><span class="o">.</span><span class="n">dtype</span><span class="p">)</span>
<span class="n">C_ref</span> <span class="o">=</span> <span class="n">C_ref</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="n">o</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BATCH</span><span class="p">,</span> <span class="n">m</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">BLOCK_OUT</span><span class="p">)</span><span class="o">.</span><span class="n">transpose</span><span class="p">((</span><span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">))</span>
<span class="n">np</span><span class="o">.</span><span class="n">testing</span><span class="o">.</span><span class="n">assert_equal</span><span class="p">(</span><span class="n">C_ref</span><span class="p">,</span> <span class="n">C_nd</span><span class="o">.</span><span class="n">numpy</span><span class="p">())</span>

<span class="c1"># Print stats</span>
<span class="k">if</span> <span class="n">env</span><span class="o">.</span><span class="n">TARGET</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;sim&quot;</span><span class="p">,</span> <span class="s2">&quot;tsim&quot;</span><span class="p">]:</span>
    <span class="n">sim_stats</span> <span class="o">=</span> <span class="n">simulator</span><span class="o">.</span><span class="n">stats</span><span class="p">()</span>
    <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Execution statistics:&quot;</span><span class="p">)</span>
    <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">sim_stats</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\t</span><span class="si">{:&lt;16}</span><span class="s2">: </span><span class="si">{:&gt;16}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="n">v</span><span class="p">))</span>

<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Successful matrix multiply test!&quot;</span><span class="p">)</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">输出:</p>
<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>Execution statistics:
        inp_load_nbytes :              256
        wgt_load_nbytes :            65536
        acc_load_nbytes :                0
        uop_load_nbytes :                8
        out_store_nbytes:              256
        gemm_counter    :              256
        alu_counter     :                0
Successful matrix multiply test!
</pre></div>
</div>
</div>
<div class="section" id="summary">
<h2>总结<a class="headerlink" href="#summary" title="永久链接至标题">¶</a></h2>
<p>This tutorial showcases the TVM workflow to implement a simple matrix
multiplication example on VTA.
The general workflow includes:</p>
<ul class="simple">
<li><p>Programming the FPGA with the VTA bitstream over RPC.</p></li>
<li><p>Describing matrix multiplication via a series of computations.</p></li>
<li><p>Describing how we want to perform the computation using schedule primitives.</p></li>
<li><p>Compiling the function to the VTA target.</p></li>
<li><p>Running the compiled module and verifying it against a numpy implementation.</p></li>
</ul>
<div class="sphx-glr-footer class sphx-glr-footer-example docutils container" id="sphx-glr-download-topic-vta-tutorials-matrix-multiply-py">
<div class="sphx-glr-download docutils container">
<p><a class="reference download internal" download="" href="../../../_downloads/de1c160863e8a3826753e987a4138298/matrix_multiply.py"><code class="xref download docutils literal notranslate"><span class="pre">Python源码下载:</span> <span class="pre">matrix_multiply.py</span></code></a></p>
</div>
<div class="sphx-glr-download docutils container">
<p><a class="reference download internal" download="" href="../../../_downloads/1ee0b869c5082223c5dfbb0fe4574252/matrix_multiply.ipynb"><code class="xref download docutils literal notranslate"><span class="pre">Jupyter</span> <span class="pre">notebook</span> <span class="pre">下载:</span> <span class="pre">matrix_multiply.ipynb</span></code></a></p>
</div>
</div>
<p class="sphx-glr-signature"><a class="reference external" href="https://sphinx-gallery.github.io">Gallery generated by Sphinx-Gallery</a></p>
</div>
</div>


           </div>
           
          </div>
          

<footer>

    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="vta_get_started.html" class="btn btn-neutral float-right" title="从 VTA 开始" accesskey="n" rel="next">下一个 <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="index.html" class="btn btn-neutral float-left" title="VTA教程" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> 上一个</a>
      
    </div>

<div id="button" class="backtop"><img src="../../../_static//img/right.svg" alt="backtop"/> </div>
<section class="footerSec">
    <div class="footerHeader">
      <ul class="d-flex align-md-items-center justify-content-between flex-column flex-md-row">
        <li class="copywrite d-flex align-items-center">
          <h5 id="copy-right-info">© 2020 Apache Software Foundation | All right reserved</h5>
        </li>
      </ul>

    </div>

    <ul>
      <li class="footernote">Copyright © 2020 The Apache Software Foundation. Apache TVM, Apache, the Apache feather, and the Apache TVM project logo are either trademarks or registered trademarks of the Apache Software Foundation.</li>
    </ul>

</section>
</footer>
        </div>
      </div>

    </section>

  </div>
  

    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js" integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q" crossorigin="anonymous"></script>
    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>

  </body>
  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
    <!-- Theme Analytics -->
    <script>
    (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
    })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

    ga('create', 'UA-75982049-2', 'auto');
    ga('send', 'pageview');
    </script>

    
   

</body>
</html>